{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Matching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. Import libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:37:59.927516500Z",
     "start_time": "2024-03-05T13:37:55.056727900Z"
    }
   },
   "outputs": [],
   "source": [
    "from lightautoml.addons.hypex import Matcher"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Create or upload your dataset  \n",
    "In this case we will create random dataset with known effect size  \n",
    "If you have your own dataset, go to the part 2 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:37:59.952659200Z",
     "start_time": "2024-03-05T13:37:59.931623200Z"
    }
   },
   "outputs": [],
   "source": [
    "from lightautoml.addons.hypex.utils.tutorial_data_creation import create_test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:18.298652Z",
     "start_time": "2024-03-05T13:37:59.946835700Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>signup_month</th>\n",
       "      <th>treat</th>\n",
       "      <th>pre_spends</th>\n",
       "      <th>post_spends</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>industry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>504.5</td>\n",
       "      <td>422.777778</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>Logistics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>500.0</td>\n",
       "      <td>506.333333</td>\n",
       "      <td>51.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>E-commerce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>485.0</td>\n",
       "      <td>434.000000</td>\n",
       "      <td>56.0</td>\n",
       "      <td>F</td>\n",
       "      <td>Logistics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>452.0</td>\n",
       "      <td>468.111111</td>\n",
       "      <td>46.0</td>\n",
       "      <td>M</td>\n",
       "      <td>E-commerce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>488.5</td>\n",
       "      <td>420.111111</td>\n",
       "      <td>56.0</td>\n",
       "      <td>M</td>\n",
       "      <td>Logistics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>9995</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>482.0</td>\n",
       "      <td>501.666667</td>\n",
       "      <td>31.0</td>\n",
       "      <td>M</td>\n",
       "      <td>Logistics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>9996</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>453.0</td>\n",
       "      <td>406.888889</td>\n",
       "      <td>53.0</td>\n",
       "      <td>M</td>\n",
       "      <td>Logistics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>9997</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>461.0</td>\n",
       "      <td>415.111111</td>\n",
       "      <td>52.0</td>\n",
       "      <td>F</td>\n",
       "      <td>E-commerce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>9998</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>491.5</td>\n",
       "      <td>439.222222</td>\n",
       "      <td>22.0</td>\n",
       "      <td>M</td>\n",
       "      <td>E-commerce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>9999</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>481.0</td>\n",
       "      <td>517.222222</td>\n",
       "      <td>53.0</td>\n",
       "      <td>M</td>\n",
       "      <td>E-commerce</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id  signup_month  treat  pre_spends  post_spends   age gender  \\\n",
       "0           0             0      0       504.5   422.777778   NaN      F   \n",
       "1           1             4      1       500.0   506.333333  51.0    NaN   \n",
       "2           2             0      0       485.0   434.000000  56.0      F   \n",
       "3           3             8      1       452.0   468.111111  46.0      M   \n",
       "4           4             0      0       488.5   420.111111  56.0      M   \n",
       "...       ...           ...    ...         ...          ...   ...    ...   \n",
       "9995     9995             2      1       482.0   501.666667  31.0      M   \n",
       "9996     9996             0      0       453.0   406.888889  53.0      M   \n",
       "9997     9997             0      0       461.0   415.111111  52.0      F   \n",
       "9998     9998            10      1       491.5   439.222222  22.0      M   \n",
       "9999     9999             2      1       481.0   517.222222  53.0      M   \n",
       "\n",
       "        industry  \n",
       "0      Logistics  \n",
       "1     E-commerce  \n",
       "2      Logistics  \n",
       "3     E-commerce  \n",
       "4      Logistics  \n",
       "...          ...  \n",
       "9995   Logistics  \n",
       "9996   Logistics  \n",
       "9997  E-commerce  \n",
       "9998  E-commerce  \n",
       "9999  E-commerce  \n",
       "\n",
       "[10000 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = create_test_data(num_users=10000, rs=42, na_step=45, nan_cols=['age', 'gender'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:18.418303900Z",
     "start_time": "2024-03-05T13:38:18.298652Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'signup_month', 'treat', 'pre_spends', 'post_spends', 'age',\n",
       "       'gender', 'industry'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:18.477754900Z",
     "start_time": "2024-03-05T13:38:18.336440500Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "treat\n",
       "0    5002\n",
       "1    4998\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['treat'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:18.619416200Z",
     "start_time": "2024-03-05T13:38:18.362065400Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "223"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['gender'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Matching  \n",
    "### 2.0 Init params\n",
    "info_col used to define informative attributes that should not be part of matching, such as user_id  \n",
    "But to explicitly store this column in the table, so that you can compare directly after computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:18.634761700Z",
     "start_time": "2024-03-05T13:38:18.408068600Z"
    }
   },
   "outputs": [],
   "source": [
    "info_col = ['user_id']\n",
    "\n",
    "outcome = 'post_spends'\n",
    "treatment = 'treat'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Simple matching\n",
    "This is the easiest way to initialize and calculate metrics on a Matching task  \n",
    "Use it when you are clear about each attribute or if you don't have any additional task conditions (Strict equality for certain features) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:18.660452500Z",
     "start_time": "2024-03-05T13:38:18.428454200Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[18.12.2024 22:05:15 | hypex | INFO]: Number of NaN values filled with zeros: 446\n"
     ]
    }
   ],
   "source": [
    "# Standard model with base parameters\n",
    "model = Matcher(input_data=df, outcome=outcome, treatment=treatment, info_col=info_col,\n",
    "                algo='fast')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "**Feature selection** models the significance of features for the accuracy of target approximation. However, it does not\n",
    "rule out the possibility of overlooked features, the complex impact of features on target description, or the\n",
    "significance of features from a business logic perspective. The algorithm will not function correctly if there are data\n",
    "leaks. \n",
    "Points to consider when selecting features:\n",
    "\n",
    "* Data leaks - these should not be present.\n",
    "* Influence on treatment distribution - features should not affect the treatment distribution.\n",
    "* The target should be describable by features.\n",
    "* All features significantly affecting the target should be included.\n",
    "* The business rationale of features.\n",
    "* The feature selection function can be useful for addressing these tasks, but it does not solve them nor does it\n",
    "  absolve the user of the responsibility for their selection, nor does it justify it.\n",
    "\n",
    "[Link to ReadTheDocs](https://hypex.readthedocs.io/en/latest/pages/modules/selectors.html#selector-classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:28.156006400Z",
     "start_time": "2024-03-05T13:38:18.489973100Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tikhomirov/PycharmProjects/Sber_New/LightAutoML/.venv/lib/python3.10/site-packages/hypex/selectors/feature_selector.py:42: UserWarning: FeatureSelector does not rule out the possibility of overlooked features, the complex impact of features on target description, or the significance of features from a business logic perspective.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>signup_month</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pre_spends</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender_F</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender_M</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>industry_Logistics</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    rank\n",
       "signup_month           1\n",
       "pre_spends             2\n",
       "age                    3\n",
       "gender_F               4\n",
       "gender_M               5\n",
       "industry_Logistics     6"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features = model.feature_select()\n",
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:28.228174700Z",
     "start_time": "2024-03-05T13:38:28.149105600Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['signup_month', 'pre_spends', 'age', 'gender_F'], dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_features = selected_features[:4].index\n",
    "chosen_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:30.559294800Z",
     "start_time": "2024-03-05T13:38:28.180655600Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8df4c3cc13a49da8f0ad88547e701b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[18.12.2024 22:05:18 | Faiss hypex | INFO]: The entry of bias into the ATT is 0.1%\n"
     ]
    }
   ],
   "source": [
    "results, quality_results, df_matched = model.estimate(features=chosen_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "### What is necessary to check in this table?¶\n",
    "* effect size in ATT - it is effect in treated group\n",
    "* standard error shows how accurately the parameter estimate corresponds to the true value in the total population\n",
    "* p-value shows the measure of randomness of the sample (in this example: A p-value of 0.0 means that there is a 0% (percentage probability) that the result is due to randomness)\n",
    "* ci (confidence interval) - the interval that covers the estimated parameter (ATT, ATC, ATE) with a given probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:30.633496200Z",
     "start_time": "2024-03-05T13:38:30.561295200Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>effect_size</th>\n",
       "      <th>std_err</th>\n",
       "      <th>p-val</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ATE</th>\n",
       "      <td>82.636764</td>\n",
       "      <td>2.339283</td>\n",
       "      <td>0.0</td>\n",
       "      <td>78.051770</td>\n",
       "      <td>87.221757</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC</th>\n",
       "      <td>101.687566</td>\n",
       "      <td>4.571439</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.727546</td>\n",
       "      <td>110.647585</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATT</th>\n",
       "      <td>63.570714</td>\n",
       "      <td>0.694726</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.209051</td>\n",
       "      <td>64.932378</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     effect_size   std_err  p-val   ci_lower    ci_upper      outcome\n",
       "ATE    82.636764  2.339283    0.0  78.051770   87.221757  post_spends\n",
       "ATC   101.687566  4.571439    0.0  92.727546  110.647585  post_spends\n",
       "ATT    63.570714  0.694726    0.0  62.209051   64.932378  post_spends"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "### Variable quality_results contains:\n",
    "\n",
    "* results of psi test\n",
    "* resulnt of Kolmogorov-Smirnov test\n",
    "* results of smd\n",
    "* number of repeats \n",
    "\n",
    "**PSI (Population Stability Index)**    \n",
    "shows the difference between treated and untreated populations\n",
    "\n",
    "**Rules:**\n",
    "\n",
    "* PSI < 0.1 - No change. You can continue using existing model.\n",
    "* PSI >=0.1 but less than 0.2 - Slight change is required.\n",
    "* PSI >=0.2 - Significant change is required. Ideally, you should not use this model any more. \n",
    "\n",
    "**SMD (Standardized Mean Differences)**    \n",
    "helps to check if the balance of the groups has been reached\n",
    "\n",
    "**Rules:**\n",
    "\n",
    "* Smaller than 0.1. For a randomized trial, the smd between all the covariates should typically fall into this bucket.\n",
    "* 0.1 - 0.2. Not necessarily balanced, but small enough that people are usually not too worried about them.\n",
    "* 0.2. Values that are greater than this threshold are considered seriously imbalanced.\n",
    "\n",
    "**Repeats**   \n",
    "shows the fraction of duplicated indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:30.892217300Z",
     "start_time": "2024-03-05T13:38:30.629494400Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['psi', 'ks_test', 'smd', 'repeats'])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quality_results.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "**Kolmogorov-Smirnov test**¶\n",
    "the distribution of one sample is compared with the distribution of the second sample and it is decided whether the samples have the same or different distribution.\n",
    "\n",
    "Table shows the p-value results of the test. If p-value < 0.05 we reject the null hypothesis and we have enough evidence to say that the sample data do not have the same distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:31.100377500Z",
     "start_time": "2024-03-05T13:38:30.776682700Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match_control_to_treat</th>\n",
       "      <th>match_treat_to_control</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>1.418144e-01</td>\n",
       "      <td>2.209790e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pre_spends</th>\n",
       "      <td>2.348715e-264</td>\n",
       "      <td>3.829212e-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>signup_month</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              match_control_to_treat  match_treat_to_control\n",
       "age                     1.418144e-01            2.209790e-07\n",
       "pre_spends             2.348715e-264            3.829212e-19\n",
       "signup_month            0.000000e+00            0.000000e+00"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quality_results['ks_test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:31.164918500Z",
     "start_time": "2024-03-05T13:38:30.915062300Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>signup_month</th>\n",
       "      <th>pre_spends</th>\n",
       "      <th>age</th>\n",
       "      <th>gender_F</th>\n",
       "      <th>gender_M</th>\n",
       "      <th>industry_Logistics</th>\n",
       "      <th>signup_month_matched</th>\n",
       "      <th>pre_spends_matched</th>\n",
       "      <th>age_matched</th>\n",
       "      <th>gender_F_matched</th>\n",
       "      <th>gender_M_matched</th>\n",
       "      <th>industry_Logistics_matched</th>\n",
       "      <th>index_matched</th>\n",
       "      <th>post_spends</th>\n",
       "      <th>post_spends_matched</th>\n",
       "      <th>post_spends_matched_bias</th>\n",
       "      <th>treat</th>\n",
       "      <th>treat_matched</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>956</td>\n",
       "      <td>8</td>\n",
       "      <td>487.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>506.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[6352]</td>\n",
       "      <td>462.222222</td>\n",
       "      <td>408.333333</td>\n",
       "      <td>54.004165</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7966</td>\n",
       "      <td>5</td>\n",
       "      <td>471.5</td>\n",
       "      <td>69.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>483.5</td>\n",
       "      <td>69.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[5349]</td>\n",
       "      <td>505.222222</td>\n",
       "      <td>415.222222</td>\n",
       "      <td>90.068667</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7231</td>\n",
       "      <td>4</td>\n",
       "      <td>487.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>496.5</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[8654]</td>\n",
       "      <td>503.555556</td>\n",
       "      <td>428.777778</td>\n",
       "      <td>74.832139</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1443</td>\n",
       "      <td>1</td>\n",
       "      <td>517.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>520.5</td>\n",
       "      <td>36.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[7881]</td>\n",
       "      <td>526.111111</td>\n",
       "      <td>423.111111</td>\n",
       "      <td>103.020028</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7973</td>\n",
       "      <td>10</td>\n",
       "      <td>501.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>525.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[7333]</td>\n",
       "      <td>440.444444</td>\n",
       "      <td>416.666667</td>\n",
       "      <td>23.915112</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>2926</td>\n",
       "      <td>0</td>\n",
       "      <td>486.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>479.5</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[971]</td>\n",
       "      <td>421.777778</td>\n",
       "      <td>509.888889</td>\n",
       "      <td>89.948512</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4998</th>\n",
       "      <td>1656</td>\n",
       "      <td>0</td>\n",
       "      <td>465.5</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>460.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[6730]</td>\n",
       "      <td>417.666667</td>\n",
       "      <td>502.222222</td>\n",
       "      <td>86.443231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>2812</td>\n",
       "      <td>0</td>\n",
       "      <td>451.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>441.5</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[6610]</td>\n",
       "      <td>417.333333</td>\n",
       "      <td>521.555556</td>\n",
       "      <td>106.002291</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5000</th>\n",
       "      <td>2813</td>\n",
       "      <td>0</td>\n",
       "      <td>504.5</td>\n",
       "      <td>69.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>514.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[9319]</td>\n",
       "      <td>422.888889</td>\n",
       "      <td>528.555556</td>\n",
       "      <td>107.086127</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5001</th>\n",
       "      <td>1297</td>\n",
       "      <td>0</td>\n",
       "      <td>495.5</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>496.5</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[3004]</td>\n",
       "      <td>415.111111</td>\n",
       "      <td>532.555556</td>\n",
       "      <td>118.405412</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index  signup_month  pre_spends   age  gender_F  gender_M  \\\n",
       "0       956             8       487.0  22.0         0         1   \n",
       "1      7966             5       471.5  69.0         0         0   \n",
       "2      7231             4       487.0  62.0         1         0   \n",
       "3      1443             1       517.0  36.0         1         0   \n",
       "4      7973            10       501.0  65.0         1         0   \n",
       "...     ...           ...         ...   ...       ...       ...   \n",
       "4997   2926             0       486.0  45.0         0         0   \n",
       "4998   1656             0       465.5  39.0         1         0   \n",
       "4999   2812             0       451.0  53.0         1         0   \n",
       "5000   2813             0       504.5  69.0         0         1   \n",
       "5001   1297             0       495.5  43.0         1         0   \n",
       "\n",
       "      industry_Logistics  signup_month_matched  pre_spends_matched  \\\n",
       "0                      0                   0.0               506.0   \n",
       "1                      1                   0.0               483.5   \n",
       "2                      1                   0.0               496.5   \n",
       "3                      1                   0.0               520.5   \n",
       "4                      0                   0.0               525.0   \n",
       "...                  ...                   ...                 ...   \n",
       "4997                   0                   2.0               479.5   \n",
       "4998                   1                   2.0               460.0   \n",
       "4999                   1                   2.0               441.5   \n",
       "5000                   0                   1.0               514.0   \n",
       "5001                   1                   1.0               496.5   \n",
       "\n",
       "      age_matched  gender_F_matched  gender_M_matched  \\\n",
       "0            23.0               0.0               1.0   \n",
       "1            69.0               0.0               1.0   \n",
       "2            62.0               1.0               0.0   \n",
       "3            36.0               1.0               0.0   \n",
       "4            65.0               1.0               0.0   \n",
       "...           ...               ...               ...   \n",
       "4997         46.0               0.0               1.0   \n",
       "4998         40.0               1.0               0.0   \n",
       "4999         57.0               1.0               0.0   \n",
       "5000         65.0               0.0               1.0   \n",
       "5001         38.0               1.0               0.0   \n",
       "\n",
       "      industry_Logistics_matched index_matched  post_spends  \\\n",
       "0                            1.0        [6352]   462.222222   \n",
       "1                            1.0        [5349]   505.222222   \n",
       "2                            1.0        [8654]   503.555556   \n",
       "3                            1.0        [7881]   526.111111   \n",
       "4                            0.0        [7333]   440.444444   \n",
       "...                          ...           ...          ...   \n",
       "4997                         1.0         [971]   421.777778   \n",
       "4998                         1.0        [6730]   417.666667   \n",
       "4999                         1.0        [6610]   417.333333   \n",
       "5000                         0.0        [9319]   422.888889   \n",
       "5001                         0.0        [3004]   415.111111   \n",
       "\n",
       "      post_spends_matched  post_spends_matched_bias  treat  treat_matched  \n",
       "0              408.333333                 54.004165      1              0  \n",
       "1              415.222222                 90.068667      1              0  \n",
       "2              428.777778                 74.832139      1              0  \n",
       "3              423.111111                103.020028      1              0  \n",
       "4              416.666667                 23.915112      1              0  \n",
       "...                   ...                       ...    ...            ...  \n",
       "4997           509.888889                 89.948512      0              1  \n",
       "4998           502.222222                 86.443231      0              1  \n",
       "4999           521.555556                106.002291      0              1  \n",
       "5000           528.555556                107.086127      0              1  \n",
       "5001           532.555556                118.405412      0              1  \n",
       "\n",
       "[10000 rows x 19 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:31.487579500Z",
     "start_time": "2024-03-05T13:38:31.046548300Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>signup_month</th>\n",
       "      <th>pre_spends</th>\n",
       "      <th>age</th>\n",
       "      <th>gender_F</th>\n",
       "      <th>gender_M</th>\n",
       "      <th>industry_Logistics</th>\n",
       "      <th>signup_month_matched</th>\n",
       "      <th>pre_spends_matched</th>\n",
       "      <th>age_matched</th>\n",
       "      <th>gender_F_matched</th>\n",
       "      <th>gender_M_matched</th>\n",
       "      <th>industry_Logistics_matched</th>\n",
       "      <th>index_matched</th>\n",
       "      <th>post_spends</th>\n",
       "      <th>post_spends_matched</th>\n",
       "      <th>post_spends_matched_bias</th>\n",
       "      <th>treat</th>\n",
       "      <th>treat_matched</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>956</td>\n",
       "      <td>8</td>\n",
       "      <td>487.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>506.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[6352]</td>\n",
       "      <td>462.222222</td>\n",
       "      <td>408.333333</td>\n",
       "      <td>54.004165</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8108</td>\n",
       "      <td>7</td>\n",
       "      <td>482.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>499.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[1742]</td>\n",
       "      <td>478.333333</td>\n",
       "      <td>411.222222</td>\n",
       "      <td>67.214942</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7228</td>\n",
       "      <td>7</td>\n",
       "      <td>493.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>511.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[5653]</td>\n",
       "      <td>492.555556</td>\n",
       "      <td>419.444444</td>\n",
       "      <td>73.220665</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7968</td>\n",
       "      <td>10</td>\n",
       "      <td>512.5</td>\n",
       "      <td>64.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>540.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[4470]</td>\n",
       "      <td>436.777778</td>\n",
       "      <td>418.666667</td>\n",
       "      <td>18.261920</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1396</td>\n",
       "      <td>6</td>\n",
       "      <td>479.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>493.5</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[6721]</td>\n",
       "      <td>485.666667</td>\n",
       "      <td>408.888889</td>\n",
       "      <td>76.860750</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4990</th>\n",
       "      <td>2924</td>\n",
       "      <td>0</td>\n",
       "      <td>493.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>496.5</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[3004]</td>\n",
       "      <td>425.000000</td>\n",
       "      <td>532.555556</td>\n",
       "      <td>108.985004</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4991</th>\n",
       "      <td>313</td>\n",
       "      <td>0</td>\n",
       "      <td>484.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>480.5</td>\n",
       "      <td>34.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[7816]</td>\n",
       "      <td>426.000000</td>\n",
       "      <td>510.777778</td>\n",
       "      <td>86.703675</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4992</th>\n",
       "      <td>2919</td>\n",
       "      <td>0</td>\n",
       "      <td>490.5</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>483.5</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[2781]</td>\n",
       "      <td>420.333333</td>\n",
       "      <td>517.555556</td>\n",
       "      <td>98.940997</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>2926</td>\n",
       "      <td>0</td>\n",
       "      <td>486.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>479.5</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[971]</td>\n",
       "      <td>421.777778</td>\n",
       "      <td>509.888889</td>\n",
       "      <td>89.948512</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5001</th>\n",
       "      <td>1297</td>\n",
       "      <td>0</td>\n",
       "      <td>495.5</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>496.5</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[3004]</td>\n",
       "      <td>415.111111</td>\n",
       "      <td>532.555556</td>\n",
       "      <td>118.405412</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4964 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index  signup_month  pre_spends   age  gender_F  gender_M  \\\n",
       "0       956             8       487.0  22.0         0         1   \n",
       "5      8108             7       482.0  54.0         0         1   \n",
       "6      7228             7       493.0  35.0         1         0   \n",
       "7      7968            10       512.5  64.0         1         0   \n",
       "8      1396             6       479.0  27.0         0         0   \n",
       "...     ...           ...         ...   ...       ...       ...   \n",
       "4990   2924             0       493.0  32.0         1         0   \n",
       "4991    313             0       484.0  35.0         1         0   \n",
       "4992   2919             0       490.5  20.0         0         1   \n",
       "4997   2926             0       486.0  45.0         0         0   \n",
       "5001   1297             0       495.5  43.0         1         0   \n",
       "\n",
       "      industry_Logistics  signup_month_matched  pre_spends_matched  \\\n",
       "0                      0                   0.0               506.0   \n",
       "5                      1                   0.0               499.0   \n",
       "6                      1                   0.0               511.0   \n",
       "7                      0                   0.0               540.0   \n",
       "8                      0                   0.0               493.5   \n",
       "...                  ...                   ...                 ...   \n",
       "4990                   1                   1.0               496.5   \n",
       "4991                   0                   2.0               480.5   \n",
       "4992                   1                   2.0               483.5   \n",
       "4997                   0                   2.0               479.5   \n",
       "5001                   1                   1.0               496.5   \n",
       "\n",
       "      age_matched  gender_F_matched  gender_M_matched  \\\n",
       "0            23.0               0.0               1.0   \n",
       "5            55.0               0.0               1.0   \n",
       "6            36.0               1.0               0.0   \n",
       "7            63.0               1.0               0.0   \n",
       "8            27.0               0.0               1.0   \n",
       "...           ...               ...               ...   \n",
       "4990         38.0               1.0               0.0   \n",
       "4991         34.0               1.0               0.0   \n",
       "4992         18.0               0.0               1.0   \n",
       "4997         46.0               0.0               1.0   \n",
       "5001         38.0               1.0               0.0   \n",
       "\n",
       "      industry_Logistics_matched index_matched  post_spends  \\\n",
       "0                            1.0        [6352]   462.222222   \n",
       "5                            0.0        [1742]   478.333333   \n",
       "6                            0.0        [5653]   492.555556   \n",
       "7                            1.0        [4470]   436.777778   \n",
       "8                            1.0        [6721]   485.666667   \n",
       "...                          ...           ...          ...   \n",
       "4990                         0.0        [3004]   425.000000   \n",
       "4991                         1.0        [7816]   426.000000   \n",
       "4992                         0.0        [2781]   420.333333   \n",
       "4997                         1.0         [971]   421.777778   \n",
       "5001                         0.0        [3004]   415.111111   \n",
       "\n",
       "      post_spends_matched  post_spends_matched_bias  treat  treat_matched  \n",
       "0              408.333333                 54.004165      1              0  \n",
       "5              411.222222                 67.214942      1              0  \n",
       "6              419.444444                 73.220665      1              0  \n",
       "7              418.666667                 18.261920      1              0  \n",
       "8              408.888889                 76.860750      1              0  \n",
       "...                   ...                       ...    ...            ...  \n",
       "4990           532.555556                108.985004      0              1  \n",
       "4991           510.777778                 86.703675      0              1  \n",
       "4992           517.555556                 98.940997      0              1  \n",
       "4997           509.888889                 89.948512      0              1  \n",
       "5001           532.555556                118.405412      0              1  \n",
       "\n",
       "[4964 rows x 19 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matched[df_matched['industry_Logistics'] != df_matched['industry_Logistics_matched']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Matching with a fixed variable  \n",
    "Used when you have categorical feature(s) that you want to compare by strict equality  \n",
    "group_col is used for strict comparison of categorical features.  \n",
    "In our case there is only one attribute  \n",
    "If there are several such attributes, you should make one of them and use it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:32.786324Z",
     "start_time": "2024-03-05T13:38:31.287765900Z"
    }
   },
   "outputs": [],
   "source": [
    "group_col = \"industry\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "Also group_col might be the list. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:33.069805500Z",
     "start_time": "2024-03-05T13:38:31.361722200Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[18.12.2024 22:05:18 | hypex | INFO]: Number of NaN values filled with zeros: 446\n"
     ]
    }
   ],
   "source": [
    "model = Matcher(input_data=df, outcome=outcome, treatment=treatment,\n",
    "                info_col=info_col, group_col=group_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:39.817868900Z",
     "start_time": "2024-03-05T13:38:31.480582100Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tikhomirov/PycharmProjects/Sber_New/LightAutoML/.venv/lib/python3.10/site-packages/hypex/selectors/feature_selector.py:42: UserWarning: FeatureSelector does not rule out the possibility of overlooked features, the complex impact of features on target description, or the significance of features from a business logic perspective.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>signup_month</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pre_spends</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender_F</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender_M</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              rank\n",
       "signup_month     1\n",
       "pre_spends       2\n",
       "age              3\n",
       "gender_F         4\n",
       "gender_M         5"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features = model.feature_select()\n",
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:39.920722400Z",
     "start_time": "2024-03-05T13:38:39.824871400Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['signup_month', 'pre_spends', 'age', 'gender_F'], dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_features = selected_features[:4].index\n",
    "chosen_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:43.485335300Z",
     "start_time": "2024-03-05T13:38:39.853989400Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14cbd6d6dc204f57ad1127bfe3f70b03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[18.12.2024 22:05:22 | Faiss hypex | INFO]: The entry of bias into the ATT is 0.2%\n"
     ]
    }
   ],
   "source": [
    "results, quality_results, df_matched = model.estimate(features=chosen_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:43.575791100Z",
     "start_time": "2024-03-05T13:38:43.487564400Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>effect_size</th>\n",
       "      <th>std_err</th>\n",
       "      <th>p-val</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ATE</th>\n",
       "      <td>81.192392</td>\n",
       "      <td>1.631028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>77.995577</td>\n",
       "      <td>84.389208</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC</th>\n",
       "      <td>98.945415</td>\n",
       "      <td>3.116605</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.836869</td>\n",
       "      <td>105.053961</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATT</th>\n",
       "      <td>63.425162</td>\n",
       "      <td>0.652349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.146559</td>\n",
       "      <td>64.703765</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     effect_size   std_err  p-val   ci_lower    ci_upper      outcome\n",
       "ATE    81.192392  1.631028    0.0  77.995577   84.389208  post_spends\n",
       "ATC    98.945415  3.116605    0.0  92.836869  105.053961  post_spends\n",
       "ATT    63.425162  0.652349    0.0  62.146559   64.703765  post_spends"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:43.843138800Z",
     "start_time": "2024-03-05T13:38:43.552239500Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>signup_month</th>\n",
       "      <th>pre_spends</th>\n",
       "      <th>age</th>\n",
       "      <th>gender_F</th>\n",
       "      <th>gender_M</th>\n",
       "      <th>industry</th>\n",
       "      <th>signup_month_matched</th>\n",
       "      <th>pre_spends_matched</th>\n",
       "      <th>age_matched</th>\n",
       "      <th>gender_F_matched</th>\n",
       "      <th>gender_M_matched</th>\n",
       "      <th>industry_matched</th>\n",
       "      <th>index_matched</th>\n",
       "      <th>post_spends</th>\n",
       "      <th>post_spends_matched</th>\n",
       "      <th>post_spends_matched_bias</th>\n",
       "      <th>treat</th>\n",
       "      <th>treat_matched</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [index, signup_month, pre_spends, age, gender_F, gender_M, industry, signup_month_matched, pre_spends_matched, age_matched, gender_F_matched, gender_M_matched, industry_matched, index_matched, post_spends, post_spends_matched, post_spends_matched_bias, treat, treat_matched]\n",
       "Index: []"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matched[df_matched['industry'] != df_matched['industry_matched']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Results  \n",
    "### 3.1 ATE, ATT, ATC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:44.131617100Z",
     "start_time": "2024-03-05T13:38:43.805857500Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>effect_size</th>\n",
       "      <th>std_err</th>\n",
       "      <th>p-val</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ATE</th>\n",
       "      <td>81.192392</td>\n",
       "      <td>1.631028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>77.995577</td>\n",
       "      <td>84.389208</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC</th>\n",
       "      <td>98.945415</td>\n",
       "      <td>3.116605</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.836869</td>\n",
       "      <td>105.053961</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATT</th>\n",
       "      <td>63.425162</td>\n",
       "      <td>0.652349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.146559</td>\n",
       "      <td>64.703765</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     effect_size   std_err  p-val   ci_lower    ci_upper      outcome\n",
       "ATE    81.192392  1.631028    0.0  77.995577   84.389208  post_spends\n",
       "ATC    98.945415  3.116605    0.0  92.836869  105.053961  post_spends\n",
       "ATT    63.425162  0.652349    0.0  62.146559   64.703765  post_spends"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 SMD, PSI, KS-test, repeats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:44.682946400Z",
     "start_time": "2024-03-05T13:38:44.007221600Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['psi', 'ks_test', 'smd', 'repeats'])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quality_results.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:44.852815200Z",
     "start_time": "2024-03-05T13:38:44.132618700Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_treated</th>\n",
       "      <th>anomaly_score_treated</th>\n",
       "      <th>check_result_treated</th>\n",
       "      <th>column_untreated</th>\n",
       "      <th>anomaly_score_untreated</th>\n",
       "      <th>check_result_untreated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age_treated</td>\n",
       "      <td>0.01</td>\n",
       "      <td>OK</td>\n",
       "      <td>age_untreated</td>\n",
       "      <td>0.08</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gender_F_treated</td>\n",
       "      <td>0.00</td>\n",
       "      <td>OK</td>\n",
       "      <td>gender_F_untreated</td>\n",
       "      <td>0.00</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>industry_treated</td>\n",
       "      <td>0.00</td>\n",
       "      <td>OK</td>\n",
       "      <td>industry_untreated</td>\n",
       "      <td>0.00</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pre_spends_treated</td>\n",
       "      <td>0.61</td>\n",
       "      <td>NOK</td>\n",
       "      <td>pre_spends_untreated</td>\n",
       "      <td>0.16</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>signup_month_treated</td>\n",
       "      <td>16.14</td>\n",
       "      <td>NOK</td>\n",
       "      <td>signup_month_untreated</td>\n",
       "      <td>0.00</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         column_treated  anomaly_score_treated check_result_treated  \\\n",
       "0           age_treated                   0.01                   OK   \n",
       "1      gender_F_treated                   0.00                   OK   \n",
       "2      industry_treated                   0.00                   OK   \n",
       "3    pre_spends_treated                   0.61                  NOK   \n",
       "4  signup_month_treated                  16.14                  NOK   \n",
       "\n",
       "         column_untreated  anomaly_score_untreated check_result_untreated  \n",
       "0           age_untreated                     0.08                     OK  \n",
       "1      gender_F_untreated                     0.00                     OK  \n",
       "2      industry_untreated                     0.00                     OK  \n",
       "3    pre_spends_untreated                     0.16                     OK  \n",
       "4  signup_month_untreated                     0.00                     OK  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quality_results['psi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:44.968735600Z",
     "start_time": "2024-03-05T13:38:44.251169500Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match_control_to_treat</th>\n",
       "      <th>match_treat_to_control</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>1.292517e-01</td>\n",
       "      <td>2.721265e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pre_spends</th>\n",
       "      <td>4.164710e-263</td>\n",
       "      <td>1.725086e-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>signup_month</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              match_control_to_treat  match_treat_to_control\n",
       "age                     1.292517e-01            2.721265e-04\n",
       "pre_spends             4.164710e-263            1.725086e-24\n",
       "signup_month            0.000000e+00            0.000000e+00"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quality_results['ks_test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:38:45.035373500Z",
     "start_time": "2024-03-05T13:38:44.332403300Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'match_control_to_treat': 0.42, 'match_treat_to_control': 0.07}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quality_results['repeats']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:39:02.572116900Z",
     "start_time": "2024-03-05T13:39:02.402954500Z"
    }
   },
   "outputs": [],
   "source": [
    "model.save(\"test_model.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:39:02.657280200Z",
     "start_time": "2024-03-05T13:39:02.525218900Z"
    }
   },
   "outputs": [],
   "source": [
    "model2 = Matcher.load(\"test_model.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:39:02.725146Z",
     "start_time": "2024-03-05T13:39:02.591885500Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>effect_size</th>\n",
       "      <th>std_err</th>\n",
       "      <th>p-val</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ATE</th>\n",
       "      <td>81.192392</td>\n",
       "      <td>1.631028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>77.995577</td>\n",
       "      <td>84.389208</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC</th>\n",
       "      <td>98.945415</td>\n",
       "      <td>3.116605</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.836869</td>\n",
       "      <td>105.053961</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATT</th>\n",
       "      <td>63.425162</td>\n",
       "      <td>0.652349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.146559</td>\n",
       "      <td>64.703765</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     effect_size   std_err  p-val   ci_lower    ci_upper      outcome\n",
       "ATE    81.192392  1.631028    0.0  77.995577   84.389208  post_spends\n",
       "ATC    98.945415  3.116605    0.0  92.836869  105.053961  post_spends\n",
       "ATT    63.425162  0.652349    0.0  62.146559   64.703765  post_spends"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2.results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-05T13:39:02.786620200Z",
     "start_time": "2024-03-05T13:39:02.626950900Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>effect_size</th>\n",
       "      <th>std_err</th>\n",
       "      <th>p-val</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ATE</th>\n",
       "      <td>81.192392</td>\n",
       "      <td>1.631028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>77.995577</td>\n",
       "      <td>84.389208</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC</th>\n",
       "      <td>98.945415</td>\n",
       "      <td>3.116605</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.836869</td>\n",
       "      <td>105.053961</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATT</th>\n",
       "      <td>63.425162</td>\n",
       "      <td>0.652349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.146559</td>\n",
       "      <td>64.703765</td>\n",
       "      <td>post_spends</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     effect_size   std_err  p-val   ci_lower    ci_upper      outcome\n",
       "ATE    81.192392  1.631028    0.0  77.995577   84.389208  post_spends\n",
       "ATC    98.945415  3.116605    0.0  92.836869  105.053961  post_spends\n",
       "ATT    63.425162  0.652349    0.0  62.146559   64.703765  post_spends"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
